// Downloader.cs
// 
// Copyright © 2009 FreeZzaph
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.
//

using System;
using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading;

namespace LibFreeZzaph
{
	
	internal class Downloader
	{
		private class DownloadItem
		{
			public string Referer { get; set; }
			public Uri Url { get; set; }
		}
		
		private class DownloadData
		{
			public IList<DownloadItem> UrlList { get; set; }
			public int FileNumberStart { get; set; }
			public int TaskNumber { get; set; }
		}
		
		private static string GenerateFileName(string fileNameFormat, IDictionary<string, string> filenameReplacementValues)
		{
			string fileName = fileNameFormat;

			foreach (string key in filenameReplacementValues.Keys)
			{
				fileName = fileName.Replace("%" + key + "%", filenameReplacementValues[key]);
			}
			
			return fileName;
		}
		
		private bool contentDownloadDone = false;
		
		private Queue<IList<DownloadItem>> downloadQueue;
		private object queueLock = new object();
		
		private List<string> fileList = new List<string>();
		private object fileListLock = new object();
		
		private IList<string> activeHosts = new List<string>();
		private object activeLock = new object();
		
		private int activeDownloaderCount = 0;
		private object downloaderLock = new object();
		
		private void FileDownloader(object state)
		{
			DownloadData dd = (DownloadData)state;
			
			List<Uri> duplicateCheck = new List<Uri>();
			int fileNumber = dd.FileNumberStart;
			string host = dd.UrlList[0].Url.Host.ToLower();
			
			Console.WriteLine(string.Format("(Task {0}) Downloading {1} files from {2}...",
				dd.TaskNumber, dd.UrlList.Count, host));
			foreach (DownloadItem item in dd.UrlList)
			{
				if (duplicateCheck.Contains(item.Url))
				{
					continue;
				}
				else
					duplicateCheck.Add(item.Url);
				
				try
				{
					byte[] fileBytes = FreeZzaphUtility.FetchUrlBytes(item.Url, item.Referer);
					
					IDictionary<string, string> filenameReplacementValues = new Dictionary<string, string>();
					
					string filePath = item.Url.AbsolutePath;

					string fileCount = string.Format("{0:00000}", fileNumber++);
					filenameReplacementValues.Add("filenumber", fileCount); 
					
					string fileExtension = filePath.Substring(filePath.LastIndexOf('.') + 1).ToLower();
					filenameReplacementValues.Add("extension", fileExtension);
					
					string origFileName = filePath.Substring(filePath.LastIndexOf('/') + 1).ToLower();
					filenameReplacementValues.Add("originalfilename", origFileName);
					
					filenameReplacementValues.Add("hostname", item.Url.Host);
					
					string fileFormat = FreeZzaphCLI.FreeZzaphCLI.Settings["SaveFileFormat"];
					string fileName = GenerateFileName(fileFormat, filenameReplacementValues);
					string saveFile = Path.Combine(SavePath, fileName);
					
					FileStream fs = File.Create(saveFile);
					fs.Write(fileBytes, 0, fileBytes.Length);
					fs.Close();
					
					lock (fileListLock)
						fileList.Add(saveFile);
					
				}
				catch
				{
				}
			}
			
			Console.WriteLine(string.Format("(Task {0}) Done downloading from {1}.", dd.TaskNumber, host)); 
			
			lock (activeLock)
				activeHosts.Remove(host);
			lock (downloaderLock)
				activeDownloaderCount--;
		}
		
		private void FileDownloadWorker()
		{
			Queue<IList<DownloadItem>> waitQueue = new Queue<IList<DownloadItem>>();
			
			int fileNumber = 0;
			int task = 0;
			while (!contentDownloadDone || downloadQueue.Count > 0 || waitQueue.Count > 0)
			{
				// If there are more than five active downloaders, take a little break.
				int active;
				lock (downloaderLock)
					active = activeDownloaderCount;
				if (active > 5)
				{
					Thread.Sleep(500);
					continue;
				}
				
				// Get the front of the download queue
				IList<DownloadItem> urls = null;
				lock (queueLock)
				{
					if (downloadQueue.Count > 0)
					{
						 urls = downloadQueue.Dequeue();
					}
				}
				
				// If there was an item in the queue..
				if (urls != null) {
					
					// An empty list causes a restart of the loop
					if (urls.Count == 0)
						continue;
					
					// Check if there's an active download on the given host
					// for the time being. If so, add the download to the
					// wait queue.
					string hostName = urls[0].Url.Host.ToLower();
					bool wasActive = false;
					lock (activeLock)
					{
						if (!activeHosts.Contains(hostName))
						{
							activeHosts.Add(hostName);
						}
						else
						{
							waitQueue.Enqueue(urls);
							wasActive = true;
						}
					}
					
					// If the host is not active, start the download
					if (!wasActive) {
						DownloadData dd = new DownloadData();
						dd.UrlList = urls;
						dd.FileNumberStart = fileNumber;
						dd.TaskNumber = task++;
						fileNumber += urls.Count;
						
						lock (downloaderLock)
							activeDownloaderCount++;
						ThreadPool.QueueUserWorkItem(FileDownloader, dd);
					}
				}
				
				if (waitQueue.Count > 0)
				{
					urls = waitQueue.Dequeue();
					
					// Check if there's an active download on the given host
					// for the time being. If so, add the download to the
					// back of the wait queue.
					string hostName = urls[0].Url.Host.ToLower();
					bool wasActive = false;
					lock (activeLock)
					{
						if (!activeHosts.Contains(hostName))
						{
							activeHosts.Add(hostName);
						}
						else
						{
							waitQueue.Enqueue(urls);
							wasActive = true;
						}
					}
					
					if (!wasActive)
					{
						DownloadData dd = new DownloadData();
						dd.UrlList = urls;
						dd.FileNumberStart = fileNumber;
						dd.TaskNumber = task++;
						fileNumber += urls.Count;
						
						lock (downloaderLock)
							activeDownloaderCount++;
						ThreadPool.QueueUserWorkItem(FileDownloader, dd);
					}
				}
				
				// To avoid busy waiting, sleep for a little while.
				Thread.Sleep(500);
			}
		}
		
		public void DownloadFrom(IList<Uri> urls, string filter)
		{
			contentDownloadDone = false;
			downloadQueue = new Queue<IList<DownloadItem>>();

			// Launch our file download worker thread
			Thread t = new Thread(FileDownloadWorker);
			t.Start();
			
			List<string> blackList = new List<string>();
			
			int linkCount = 0;
			List<Uri> duplicateCheck = new List<Uri>();
			
			int urlCount = urls.Count, currentUrl = 1;
			foreach (Uri url in urls)
			{
				string host = url.Host;
				
				if (blackList.Contains(host.ToLower()))
				{
					Console.WriteLine(string.Format("Skipping blacklisted address. ({0}/{1})",
						currentUrl, --urlCount));
					continue;
				}
				
				if (duplicateCheck.Contains(url))
				{
					Console.WriteLine(string.Format("Skipping duplicate address. ({0}/{1})",
						currentUrl, --urlCount));
					continue;
				}
				
				Console.WriteLine(string.Format("Gathering URLs from {0} ({1}/{2}).",
					url.ToString(), currentUrl, urlCount));
				duplicateCheck.Add(url);
				
				try 
				{
					IList<Uri> fileUrls = CollectUrlsFrom(url, filter);
					Console.WriteLine(string.Format("Found {0} URLs at {1}.", fileUrls.Count, url.ToString()));
					linkCount += fileUrls.Count;
					
					IList<DownloadItem> itemList = new List<DownloadItem>();
					foreach (Uri fileUrl in fileUrls)
					{
						DownloadItem item = new DownloadItem();
						item.Referer = url.ToString();
						item.Url = fileUrl;
						
						itemList.Add(item);
					}
					
					lock (queueLock)						
						downloadQueue.Enqueue(itemList);
					
					currentUrl++;
				}
				catch
				{
					Console.WriteLine("failed! Blacklisting address.");
					blackList.Add(host);
					urlCount--;
				}	
			}
			Console.WriteLine(string.Format("Found a total of {0} files to download.", linkCount));
			Console.WriteLine();
			
			contentDownloadDone = true;
			t.Join();
			
			while (true)
			{
				int active;
				lock (downloaderLock)
					active = activeDownloaderCount;
				
				if (active == 0)
					break;
				else
					Thread.Sleep(500);
			}
		}
		
		private IList<Uri> CollectUrlsFrom(Uri url, string filter)
		{
			string page = FreeZzaphUtility.FetchUrl(url);
			List<Uri> urlList = new List<Uri>();
			Regex filterPattern = new Regex(@"<a.+?href=""?([^"">]+?\.(?:" + filter + @"))""?[^>]*?>", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Singleline);
			
			Match match = filterPattern.Match(page);
			while (match.Success)
			{
				try
				{
					Uri fileUrl = new Uri(url, match.Groups[1].ToString());
					urlList.Add(fileUrl);
				}
				catch
				{
					// Malformed URL, ignore
				}
				
				match = match.NextMatch();
			}
			
			return urlList;
		}
		
		public IList<string> Files
		{
			get
			{
				return fileList;
			}
		}
		
		public string SavePath
		{
			get; set;
		}
		
	}
}
